package com.anmai.crawler1.service;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.anmai.crawler1.base.entity.BaseBean;
import com.anmai.crawler1.dao.AreaDao;
import com.anmai.crawler1.dao.BussinessDao;
import com.anmai.crawler1.dao.CityDao;
import com.anmai.crawler1.dao.ShopDao;
import com.anmai.crawler1.entity.*;
import com.anmai.crawler1.utils.JsoupUtils;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.*;

@Service("crawlerService")
public class CrawlerService {

    private final Logger logger = LoggerFactory.getLogger(CrawlerService.class);

    @Autowired
    private CityDao cityDao;

    @Autowired
    private AreaDao areaDao;

    @Autowired
    private BussinessDao bussinessDao;

    @Autowired
    private ShopDao shopDao;


    /**
     * 获取所有的城市
     *
     */
   public void insertCities(){
       try {
           Document document = JsoupUtils.getDocument("http://www.meituan.com/changecity/");
           //获取所有的城市首字母分类
           Elements cities = document.body().getElementsByClass("cities");
           //遍历每个字母中的城市
           cities.parallelStream().forEach(element -> {
               CityBean cityBean = new CityBean();
               cityBean.setUpdateTime(new Date());
               cityBean.setCreateTime(new Date());
               //拿到每个a标签中的内容
               Elements aList = element.getElementsByTag("a");
               //获取首字母
               String spell = element.previousElementSibling().text();
               cityBean.setInitial(spell);

               //遍历每个城市
               aList.parallelStream().forEach(element1 -> {
                   logger.info(element1.text() + "========" + element1.attr("href"));
                   cityBean.setCityName(element1.text());
                   cityBean.setUrl("http:"+element1.attr("href"));
                   //数据存储到mysql
                   if(cityDao.isExist(cityBean) == 0)
                     cityDao.insert(cityBean);
               });
           });
       } catch (Exception e) {
           logger.error("ERROR:\t获取城市列表报错啦。。。。。");
           e.printStackTrace();
       }
   }

    /**
     *  获取城市中的区，商圈以及商圈信息
     * @param cityBeans  需要处理的city集合
     */
    public void insertArea(List<CityBean> cityBeans){
        try {
            //城市列表
            /*List<CityBean> cityBeans = new ArrayList<>();
            cityBeans.addAll(cityDao.findCitiesByStatus(cityStatus));*/
            cityBeans.stream().forEach(cityBean -> {
                AreaBean areaBean = new AreaBean(); areaBean.setCreateTime(new Date());
                BussinessBean bussinessBean = new BussinessBean(); bussinessBean.setCreateTime(new Date());
                //获取该城市的区，商圈网页源码
                try {
                    JSONObject jsonObject =CrawlerService.getJson(cityBean.getUrl() + "/meishi/");
                    //将json转为对象处理
                    JSONArray areas = jsonObject.getJSONObject("filters").getJSONArray("areas");
                    //判断是否含有区域
                    for (Iterator iterator = areas.iterator(); iterator.hasNext();) {
                        //解析区域
                        JSONObject area = (JSONObject) iterator.next();
                        areaBean.setCityId(cityBean.getId());
                        areaBean.setAreaName(area.getString("name"));
                        areaBean.setUrl(area.getString("url"));
                        areaBean.setStatus(1);
                        //判断是否存在
                        if(areaDao.isExist(areaBean) == 0){
                            areaDao.insert(areaBean);
                            logger.info("Success:insertArea:"+areaBean.getAreaName()+"=========="+areaBean.getUrl());
                        }

                        JSONArray subAreas = area.getJSONArray("subAreas");
                        for (Iterator iterator1 = subAreas.iterator();iterator1.hasNext();){
                            //解析商圈
                            JSONObject bussiness = (JSONObject) iterator1.next();
                            if(!"全部".equals(bussiness.getString("name"))){
                                bussinessBean.setAreaId(areaBean.getId());
                                bussinessBean.setBussinessName(bussiness.getString("name"));
                                bussinessBean.setUrl(bussiness.getString("url"));
                                if(bussinessDao.isExist(bussinessBean) > 0) continue;
                                bussinessDao.insert(bussinessBean);
                                logger.info("\t\t\t"+bussinessBean.getBussinessName()+"==========="+bussinessBean.getUrl());
                            }
                        }
                        //更新city状态
                        cityBean.setStatus(1);
                        cityDao.updateStatus(cityBean);
                    }
                    Thread.sleep(100);
                } catch (Exception e) {
                    logger.error("ERROR:insertArea:---------->"+cityBean.getUrl()+"================="+areaBean.getAreaName()+"==========="+bussinessBean.getBussinessName());
                    //更新city状态
                    cityBean.setStatus(-1);
                    cityDao.updateStatus(cityBean);
                    e.printStackTrace();
                }
            });

        } catch (Exception e) {
            logger.error("ERROR:insertArea:---------->");
            e.printStackTrace();
        }
    }

    //获取每个城市的商圈信息
    public void insertShop(){
        try {
            //查询没有区域的city
            List<CityBean> cityNoBussiness = cityDao.findCityNoBussiness();
            insertCityShop(cityNoBussiness);
            Thread.sleep(1000 * 60);
            //查询没有商圈的area
            List<AreaBean> areaNoBussiness = areaDao.findAreaNoBussiness();
            insertAreaShop(areaNoBussiness);
            Thread.sleep(1000 * 60);
            //查询所有商圈
            insertBussinessShop(bussinessDao.findErrorBusinesses(0));
            insertBussinessShop(bussinessDao.findErrorBusinesses(-10));

        } catch (Exception e) {
            logger.warn("Error:=====================insertShop");
            e.printStackTrace();
        }
    }



    /**
     * 保存商户详细信息
     */
    public  void insertShopDetail(){
        //获取商铺连接
        //获取总记录数
        List<ShopBean> list;
        while (true) {
            list = shopDao.selectByPageAndExceptStatus(0, 1000, 1);
            if(list == null || (list != null && list.size() == 0))  break;
            list.stream().forEach(shopBean -> {
                try {
                    JSONObject shop = CrawlerService.getShopDetailJson(shopBean.getUrl());
                    JSONObject shopDetailJson = shop.getJSONObject("detailInfo");
                    shopBean.setPhone(shopDetailJson.getString("phone"));
                    shopBean.setLat(shopDetailJson.getDouble("latitude"));
                    shopBean.setLng(shopDetailJson.getDouble("longitude"));
                    shopBean.setBrandId(shopDetailJson.getInteger("brandId"));
                    shopBean.setAvgPrice(shopDetailJson.getInteger("avgPrice")+"");
                    shopBean.setAvgScore(shopDetailJson.getDouble("avgScore"));
                    shopBean.setAllCommentnum(shopDetailJson.getInteger("avgScore"));
                    shopBean.setAddress(shopDetailJson.getString("address"));
                    shopBean.setStatus(1);
                    shopBean.setUpdateTime(new Date());
                    //获取文章的信息内容
                    String cityName = shop.getJSONArray("crumbNav").getJSONObject(0).getString("title").replaceAll("美团", "");
                    String tag = shop.getJSONArray("crumbNav").getJSONObject(2).getString("title").replaceAll(cityName, "");
                    shopBean.setTag(tag);
                    //获取city_id_origin,
                    Integer cityIdOrigin = shop.getJSONObject("$meta").getInteger("cityId");
                    shopBean.setCityIdOrigin(cityIdOrigin);
                    shopDao.updateByPrimaryKeySelective(shopBean);
                    logger.info("Success： " + shopBean.getTitle() + "===========" + shopBean.getUrl());
//                        Thread.sleep(200);
                } catch (Exception e) {
                    //當前shop詳情更新失敗
                    shopBean.setStatus(-1);
                    shopBean.setUpdateTime(new Date());
                    shopDao.updateByPrimaryKeySelective(shopBean);
                    logger.error("DownUrl ERROR" + shopBean.getTitle() + "===========" + shopBean.getUrl());
                    e.printStackTrace();
                }
            });
            try {
                Thread.sleep(1000 * 5);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }


    //获取商户的分店信息
    public void insertShopBranch(){
        List<ShopBean> list = new ArrayList<>();
        while (true) {
            list = shopDao.findHasBranch();
            if(list.isEmpty() || list.size() ==0)  break;
            list.stream().forEach(shopBean -> {
                try {
                    //默认获取二十家分店信息
                    String html = JsoupUtils.getHtml("http://www.meituan.com/meishi/api/poi/getBranch?brandId="+shopBean.getBrandId()+"&cityId="+shopBean.getCityIdOrigin()+"&offset=0&limit=20");
                    JSONObject json = JSON.parseObject(html);
                    JSONObject shopBranch = json.getJSONObject("data");
                    shopBean.setBranchDetail(shopBranch.toJSONString());
                    //保存分店信息
                    shopBean.setStatus(6);
                    shopDao.saveShopBranch(shopBean);
                    Thread.sleep(1000);
                    logger.info("SUCCESS:=========>insertShopBranch"+shopBean.getTitle()+"=======http://www.meituan.com/meishi/api/poi/getBranch?brandId="+shopBean.getBrandId()+"&cityId="+shopBean.getCityIdOrigin()+"&offset=0&limit=20");
                } catch (Exception e) {
                    //分店信息获取失败，更新状态
                    shopBean.setStatus(-6);
                    shopDao.saveShopBranch(shopBean);
                    logger.error("ERROR:===========>insertShopBranch"+shopBean.getTitle()+"=======http://www.meituan.com/meishi/api/poi/getBranch?brandId="+shopBean.getBrandId()+"&cityId="+shopBean.getCityIdOrigin()+"&offset=0&limit=20");
                    e.printStackTrace();
                }
            });
        }
    }


    //获取商户tag+评论
    public void insertShopTagAndComment(){
        while (true) {
            //获取商户的信息
            List<ShopBean> list = new ArrayList<>();
            list.addAll(shopDao.findShopByStatus(6, 0, 1000));
            list.addAll(shopDao.findShopByStatus(-6, 0, 1000));
            list.addAll(shopDao.findShopByStatus(1, 0, 1000));
            if(list.isEmpty() || list.size() == 0) break;
            list.stream().forEach(shopBean -> {
                try {
                    String html = JsoupUtils.getHtml("http://www.meituan.com/meishi/api/poi/getMerchantComment?pageSize=10&sortType=1&id=" + shopBean.getPoiId());
                    String tags = JSON.parseObject(html).getJSONObject("data").getJSONArray("tags").toJSONString();
                    String comments = JSON.parseObject(html).getJSONObject("data").getJSONArray("comments").toJSONString();
                    shopBean.setStatus(8);
                    shopBean.setCommentTags(tags);
                    shopBean.setComments(comments);
                    shopDao.updateByPrimaryKeySelective(shopBean);
                    logger.info("SUCCESS=====>insertShopTagAndComment:" + shopBean.getTitle() + "=========" + "http://www.meituan.com/meishi/api/poi/getMerchantComment?pageSize=50&sortType=1&&id=" + shopBean.getPoiId());
                    Thread.sleep(1000);
                } catch (Exception e) {
                    //评论获取失败
                    shopBean.setStatus(-8);
                    shopDao.updateByPrimaryKeySelective(shopBean);
                    logger.info("ERROR=====>insertShopTagAndComment:" + shopBean.getTitle() + "=========" + "http://www.meituan.com/meishi/api/poi/getMerchantComment?pageSize=50&sortType=1&&id=" + shopBean.getPoiId());
                    e.printStackTrace();
                }

            });
        }
    }
    //获取商家信息
    /**
     * 保存没有区域的city的商户信息
     */
    public void insertCityShop(List<CityBean> cityNoBussiness){
        //获取没有商圈的区域
//        List<CityBean> cityNoBussiness = cityDao.findCityNoBussiness();
        cityNoBussiness.stream().forEach(cityBean -> {
            //获取当前页数
            try {
                JSONObject json = CrawlerService.getJson(cityBean.getUrl()+"/meishi/");
                //保存商铺信息
                saveShop(json,cityBean,shopDao);
                //修改状态为10
                cityBean.setStatus(10);
                cityDao.updateStatus(cityBean);
                logger.info("城市完成：insertCityShop=======>"+cityBean.getCityName()+"========"+cityBean.getUrl());
                Thread.sleep(100);
            } catch (Exception e) {
                //修改area状态
                cityBean.setStatus(-10);
                cityDao.updateStatus(cityBean);
                logger.error("ERROR：insertCityShop========>"+cityBean.getCityName()+"========"+cityBean.getUrl());
                e.printStackTrace();
            }
        });

    }

    /**
     * 保存没有商圈的area的商户信息
     */
    public void insertAreaShop( List<AreaBean> areaNoBussiness ){
        //获取没有商圈的区域
//        List<AreaBean> areaNoBussiness = areaDao.findAreaNoBussiness();
        areaNoBussiness.stream().forEach(areaBean -> {
            //获取当前页数
            try {
                JSONObject json = CrawlerService.getJson(areaBean.getUrl());
                //保存商铺信息
                saveShop(json,areaBean,shopDao);
                //修改状态为10
                //修改状态为10
                areaBean.setStatus(10);
                areaDao.updateStatus(areaBean);
                logger.info("区域完成:insertAreaShop=======>"+areaBean.getAreaName()+"========"+areaBean.getUrl());

                Thread.sleep(500);
            } catch (Exception e) {
                //修改area状态
                areaBean.setStatus(-10);
                areaDao.updateStatus(areaBean);
                logger.error("ERROR:insertAreaShop========>"+areaBean.getAreaName()+"========"+areaBean.getUrl());
                e.printStackTrace();
            }
        });

    }

    /**
     * 保存商圈的商户信息
     */
    public void insertBussinessShop(List<BussinessBean> businesses){
        //获取所有的商圈信息
        businesses.stream().forEach(bussinessBean -> {
            //获取当前页数
            try {
                JSONObject json = CrawlerService.getJson(bussinessBean.getUrl());
                //保存商铺信息
                saveShop(json,bussinessBean,shopDao);
                //修改状态为10
                bussinessBean.setStatus(10);
                bussinessDao.updateStatus(bussinessBean);
                logger.info("商圈完成:insertBussinessShop=======>"+bussinessBean.getBussinessName()+"========"+bussinessBean.getUrl());

                Thread.sleep(100);
            } catch (Exception e) {
                //修改area状态
                bussinessBean.setStatus(-10);
                bussinessDao.updateStatus(bussinessBean);
                logger.error("ERROR:insertBussinessShop========>"+bussinessBean.getBussinessName()+"========"+bussinessBean.getUrl());
                e.printStackTrace();
            }
        });


    }



    //批量保存商家信息
    public static void batchSaveShop(List<ShopBean> shopBeans, ShopDao shopDao){
        shopBeans.stream().forEach(shopBean -> {
            //遍历数组，存入数据
            if(shopDao.isExist(shopBean) == 0) {
                shopDao.insertSelective(shopBean);
            }else {
                ShopBean shopBean1 = shopDao.selectByOnly(shopBean);
                shopBean.setId(shopBean1.getAreaId());
                shopBean.setUpdateTime(new Date());
                shopDao.updateByPrimaryKeySelective(shopBean);
            }
        });
    }

    /***
     * 保存商户信息
     * @param json  包含商户信息的json数据
     * @param baseBean
     * @param shopDao
     */
    public static  void saveShop(JSONObject json, BaseBean baseBean, ShopDao shopDao) throws IOException {
        //获取当前页
        Integer currPage = json.getInteger("pn");
        JSONObject poiLists = json.getJSONObject("poiLists");
        //保存当前数据
        JSONArray poiInfos = poiLists.getJSONArray("poiInfos");
        List<ShopBean> shopBeans = JSONObject.parseArray(poiInfos.toJSONString(), ShopBean.class);
        inverseShop(shopBeans,baseBean);
        batchSaveShop(shopBeans,shopDao);
        //保存下一页
        saveNextShop(poiLists,baseBean,shopDao);
    }


    /**
     * 保存含有下一页商户的信息
     *
     * @param poiLists  json对象中的poiList数组
     * @param baseBean
     * @param shopDao
     */
    public static  void  saveNextShop(JSONObject poiLists, BaseBean baseBean,ShopDao shopDao) throws IOException {
        //获取总记录数
        Integer counts = poiLists.getInteger("totalCounts");
        int pages = (counts / 32) + 1;
        if(pages > 2) {
            for (int i = 2; i <= pages; i++) {
                //获取总页数数据
                JSONObject json = CrawlerService.getJson(baseBean.getUrl() + "pn" + i + "/");
                JSONArray poiInfos = json.getJSONObject("poiLists").getJSONArray("poiInfos");
                List<ShopBean> shopBeans = JSONObject.parseArray(poiInfos.toJSONString(), ShopBean.class);
                inverseShop(shopBeans,baseBean);
                CrawlerService.batchSaveShop(shopBeans, shopDao);
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * shop对象内容处理
     * @param shopBeans
     * @param baseBean
     */
    public static  void inverseShop(List<ShopBean> shopBeans,BaseBean baseBean){
        shopBeans.stream().forEach(shopBean -> {
            if(baseBean instanceof AreaBean){
                shopBean.setCityId(((AreaBean) baseBean).getCityId());
                shopBean.setAreaId(baseBean.getId());
                shopBean.setBussinessId(-10);
                shopBean.setUrl(baseBean.getUrl().substring(0, baseBean.getUrl().indexOf("meishi"))+"meishi/"+shopBean.getPoiId()+"/");
            }else if(baseBean instanceof  BussinessBean){
                shopBean.setCityId(-10);
                shopBean.setAreaId(((BussinessBean)baseBean).getAreaId());
                shopBean.setBussinessId(baseBean.getId());
                shopBean.setUrl(baseBean.getUrl().substring(0, baseBean.getUrl().indexOf("meishi"))+"meishi/"+shopBean.getPoiId()+"/");
            }else {
                shopBean.setAreaId(-10);
                shopBean.setBussinessId(-10);
                shopBean.setCityId(baseBean.getId());
                shopBean.setUrl(baseBean.getUrl()+"/meishi/"+shopBean.getPoiId()+"/");
            }
        });

    }



    /**
     * 获取json对象
     * @param url
     * @return
     */
    public static JSONObject getJson(String url) throws IOException {
        String json = "";
        String html = JsoupUtils.getHtml(url);
        //获取到json
        String temp = html.split("window._appState =")[1];
        json = temp.split(";</script>")[0];
        return JSON.parseObject(json);
    }

    /**
     * 获取hopDetail的json对象
     * @param url
     * @return
     */
    public static JSONObject getShopDetailJson(String url) throws IOException {
        String json = "";
        String html = "";
        try {
             html = JsoupUtils.getConnection(url)
//                    .header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36")
                    .header("User-Agent", "User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36")
                    .header("Cookie","_lxsdk_cuid=162dde99aabc8-07eec9ead2e60d-3f3c5906-100200-162dde99aacc8;")
                    .ignoreContentType(true)
                    .execute()
                    .body();
        }catch (IOException e){
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e1) {
                e1.printStackTrace();
            }
            html = JsoupUtils.getConnection(url)
                    .header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36")
                    .header("Cookie","_lxsdk_cuid=162dde99aabc8-07eec9ead2e60d-3f3c5906-100200-162dde99aacc8;")
                    .ignoreContentType(true)
                    .execute()
                    .body();
        }
        //获取到json
        String temp = html.split("window._appState =")[1];
        json = temp.split(";</script>")[0];
        return JSON.parseObject(json);
    }



    /*

   //从城市中直接获area,bussiness,shop信息
    public void insertNewMethod(Integer cityStatus){
        try {
            //城市列表
            List<CityBean> cityBeans = new ArrayList<>();
            cityBeans.addAll(cityDao.findCitiesByStatus(0));
            cityBeans.stream().forEach(cityBean -> {
                AreaBean areaBean = new AreaBean(); areaBean.setCreateTime(new Date());
                BussinessBean bussinessBean = new BussinessBean(); bussinessBean.setCreateTime(new Date());
                try {
                    JSONObject json =CrawlerService.getJson(cityBean.getUrl() + "/meishi/").getJSONObject("filters");
                    //获取当前城市的city,bussiness,tags等信息

                    JSONArray cates = json.getJSONArray("cates");
                    List<MTagBean> mTagBeans = JSON.parseArray(cates.toJSONString(), MTagBean.class);

                    //判断是否有子标签
                    if (mTagBeans != null && mTagBeans.size() > 0) {
                        //有子标签,遍历子标签
                        mTagBeans.stream().forEach(mTagBean -> {
                            JSONArray areas = json.getJSONArray("areas");
                            String temp  = "/c"+mTagBean.getId();
                            List<AreaBean> areaBeans = JSON.parseArray(areas.toJSONString(), AreaBean.class);
                            //判断是否含有区域
                            if (areaBeans != null && areaBeans.size() > 0) {
                                areaBeans.stream().forEach(area -> {

                                });
                            }else {


                            }
                        });
                    }else {
                        //没有子标签

                    }


                } catch (Exception e) {
                    logger.error("ERROR:insertNewMethod:---------->"+cityBean.getUrl()+"================="+areaBean.getAreaName()+"==========="+bussinessBean.getBussinessName());
                    //更新city状态
                    cityBean.setStatus(-1);
                    cityDao.updateStatus(cityBean);
                    e.printStackTrace();
                }
            });
        } catch (Exception e) {
            logger.warn("ERROR:inserNew=====>根据城市状态查询失败");
            e.printStackTrace();
        }
    }

    //获取json中的信息
    public <T> List<T> getInformationFromJson(JSONObject json,T t){

        if (t instanceof AreaBean) {

        } else if (t instanceof BussinessBean) {

        }else {

        }
       return  null;
    }
*/


}
